{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree as etree\n",
    "import codecs\n",
    "import csv\n",
    "import time\n",
    "import os\n",
    "import re\n",
    "import gzip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_WIKI_XML = 'd:/MMD- Project - Power2TheWiki/data/en/'\n",
    "PATH_WIKI_OUT = 'd:/MGontar/Storages/Google Drive/20. Data Science/0.UCU Study/Mining Massive Datasets/Project/out/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "WIKI_FILENAMES = []\n",
    "for file in os.listdir(PATH_WIKI_XML):\n",
    "    if file.endswith(\".xml\"):\n",
    "        WIKI_FILENAMES.append(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENCODING = \"utf-8\"\n",
    "\n",
    "# Nicely formatted time string\n",
    "def hms_string(sec_elapsed):\n",
    "    h = int(sec_elapsed / (60 * 60))\n",
    "    m = int((sec_elapsed % (60 * 60)) / 60)\n",
    "    s = sec_elapsed % 60\n",
    "    return \"{}:{:>02}:{:>05.2f}\".format(h, m, s)\n",
    "\n",
    "\n",
    "def strip_tag_name(t):\n",
    "    t = elem.tag\n",
    "    idx = k = t.rfind(\"}\")\n",
    "    if idx != -1:\n",
    "        t = t[idx + 1:]\n",
    "    return t\n",
    "\n",
    "\n",
    "totalCount = 0\n",
    "articleCount = 0\n",
    "redirectCount = 0\n",
    "total_article_text_len = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File processed: enwiki-20180620-pages-meta-current01-p10p30303.xml\n",
      "Total pages: 19,723\n",
      "Article pages: 15,048\n",
      "Redirect pages: 4,675\n",
      "Total article lenght: 584,538,815\n",
      "Elapsed time: 0:01:02.85\n",
      "File processed: enwiki-20180620-pages-meta-current02-p30304p88444.xml\n",
      "Total pages: 58,268\n",
      "Article pages: 41,886\n",
      "Redirect pages: 16,382\n",
      "Total article lenght: 1,276,094,074\n",
      "Elapsed time: 0:02:25.05\n",
      "100,000\n",
      "File processed: enwiki-20180620-pages-meta-current03-p88445p200507.xml\n",
      "Total pages: 140,186\n",
      "Article pages: 107,410\n",
      "Redirect pages: 32,776\n",
      "Total article lenght: 2,392,075,343\n",
      "Elapsed time: 0:04:31.90\n",
      "200,000\n",
      "File processed: enwiki-20180620-pages-meta-current04-p200511p352689.xml\n",
      "Total pages: 223,644\n",
      "Article pages: 156,525\n",
      "Redirect pages: 67,119\n",
      "Total article lenght: 3,282,443,066\n",
      "Elapsed time: 0:07:00.31\n",
      "300,000\n",
      "File processed: enwiki-20180620-pages-meta-current05-p352690p565313.xml\n",
      "Total pages: 343,093\n",
      "Article pages: 221,518\n",
      "Redirect pages: 121,575\n",
      "Total article lenght: 4,243,772,308\n",
      "Elapsed time: 0:09:12.11\n",
      "400,000\n",
      "500,000\n",
      "File processed: enwiki-20180620-pages-meta-current06-p565314p892912.xml\n",
      "Total pages: 520,145\n",
      "Article pages: 299,718\n",
      "Redirect pages: 220,427\n",
      "Total article lenght: 5,268,553,043\n",
      "Elapsed time: 0:11:38.20\n",
      "600,000\n",
      "File processed: enwiki-20180620-pages-meta-current07-p892914p1268691.xml\n",
      "Total pages: 688,258\n",
      "Article pages: 390,556\n",
      "Redirect pages: 297,702\n",
      "Total article lenght: 6,293,147,520\n",
      "Elapsed time: 0:14:52.65\n",
      "700,000\n",
      "800,000\n",
      "900,000\n",
      "File processed: enwiki-20180620-pages-meta-current08-p1268692p1791079.xml\n",
      "Total pages: 906,231\n",
      "Article pages: 495,668\n",
      "Redirect pages: 410,563\n",
      "Total article lenght: 7,378,662,910\n",
      "Elapsed time: 0:18:07.46\n",
      "1,000,000\n",
      "1,100,000\n",
      "File processed: enwiki-20180620-pages-meta-current09-p1791080p2336422.xml\n",
      "Total pages: 1,133,542\n",
      "Article pages: 602,494\n",
      "Redirect pages: 531,048\n",
      "Total article lenght: 8,424,560,411\n",
      "Elapsed time: 0:20:55.02\n",
      "1,200,000\n",
      "1,300,000\n",
      "File processed: enwiki-20180620-pages-meta-current10-p2336425p3046511.xml\n",
      "Total pages: 1,384,129\n",
      "Article pages: 725,657\n",
      "Redirect pages: 658,472\n",
      "Total article lenght: 9,473,273,537\n",
      "Elapsed time: 0:23:58.13\n",
      "1,400,000\n",
      "1,500,000\n",
      "1,600,000\n",
      "File processed: enwiki-20180620-pages-meta-current11-p3046514p3926861.xml\n",
      "Total pages: 1,666,031\n",
      "Article pages: 858,901\n",
      "Redirect pages: 807,130\n",
      "Total article lenght: 10,565,017,949\n",
      "Elapsed time: 0:27:17.99\n",
      "1,700,000\n",
      "1,800,000\n",
      "1,900,000\n",
      "File processed: enwiki-20180620-pages-meta-current12-p3926863p5040436.xml\n",
      "Total pages: 1,983,434\n",
      "Article pages: 1,004,366\n",
      "Redirect pages: 979,068\n",
      "Total article lenght: 11,679,401,730\n",
      "Elapsed time: 0:30:59.87\n",
      "2,000,000\n",
      "2,100,000\n",
      "2,200,000\n",
      "2,300,000\n",
      "File processed: enwiki-20180620-pages-meta-current13-p5040438p6197594.xml\n",
      "Total pages: 2,314,526\n",
      "Article pages: 1,152,258\n",
      "Redirect pages: 1,162,268\n",
      "Total article lenght: 12,747,450,418\n",
      "Elapsed time: 0:34:43.43\n",
      "2,400,000\n",
      "2,500,000\n",
      "2,600,000\n",
      "File processed: enwiki-20180620-pages-meta-current14-p6197598p7697598.xml\n",
      "Total pages: 2,667,351\n",
      "Article pages: 1,309,839\n",
      "Redirect pages: 1,357,512\n",
      "Total article lenght: 13,800,819,769\n",
      "Elapsed time: 0:38:46.67\n",
      "File processed: enwiki-20180620-pages-meta-current14-p7697598p7744799.xml\n",
      "Total pages: 2,677,460\n",
      "Article pages: 1,314,391\n",
      "Redirect pages: 1,363,069\n",
      "Total article lenght: 13,830,940,055\n",
      "Elapsed time: 0:38:55.29\n",
      "2,700,000\n",
      "2,800,000\n",
      "2,900,000\n",
      "3,000,000\n",
      "File processed: enwiki-20180620-pages-meta-current15-p7744803p9244803.xml\n",
      "Total pages: 3,013,534\n",
      "Article pages: 1,455,029\n",
      "Redirect pages: 1,558,505\n",
      "Total article lenght: 14,783,726,204\n",
      "Elapsed time: 0:42:49.18\n",
      "File processed: enwiki-20180620-pages-meta-current15-p9244803p9518048.xml\n",
      "Total pages: 3,071,124\n",
      "Article pages: 1,479,354\n",
      "Redirect pages: 1,591,770\n",
      "Total article lenght: 14,947,365,270\n",
      "Elapsed time: 0:43:30.24\n",
      "3,100,000\n",
      "File processed: enwiki-20180620-pages-meta-current16-p11018050p11539266.xml\n",
      "Total pages: 3,178,869\n",
      "Article pages: 1,525,730\n",
      "Redirect pages: 1,653,139\n",
      "Total article lenght: 15,247,838,550\n",
      "Elapsed time: 0:44:51.64\n",
      "3,200,000\n",
      "3,300,000\n",
      "3,400,000\n",
      "File processed: enwiki-20180620-pages-meta-current16-p9518050p11018050.xml\n",
      "Total pages: 3,453,684\n",
      "Article pages: 1,645,080\n",
      "Redirect pages: 1,808,604\n",
      "Total article lenght: 16,038,584,236\n",
      "Elapsed time: 0:48:41.88\n",
      "3,500,000\n",
      "3,600,000\n",
      "3,700,000\n",
      "3,800,000\n",
      "File processed: enwiki-20180620-pages-meta-current17-p11539268p13039268.xml\n",
      "Total pages: 3,840,604\n",
      "Article pages: 1,817,793\n",
      "Redirect pages: 2,022,811\n",
      "Total article lenght: 16,973,390,620\n",
      "Elapsed time: 0:52:51.06\n",
      "3,900,000\n",
      "File processed: enwiki-20180620-pages-meta-current17-p13039268p13693071.xml\n",
      "Total pages: 3,993,112\n",
      "Article pages: 1,880,482\n",
      "Redirect pages: 2,112,630\n",
      "Total article lenght: 17,315,259,388\n",
      "Elapsed time: 0:54:30.70\n",
      "4,000,000\n",
      "4,100,000\n",
      "4,200,000\n",
      "4,300,000\n",
      "File processed: enwiki-20180620-pages-meta-current18-p13693074p15193074.xml\n",
      "Total pages: 4,359,000\n",
      "Article pages: 2,016,460\n",
      "Redirect pages: 2,342,540\n",
      "Total article lenght: 18,151,211,430\n",
      "Elapsed time: 0:59:08.60\n",
      "4,400,000\n",
      "4,500,000\n",
      "File processed: enwiki-20180620-pages-meta-current18-p15193074p16120542.xml\n",
      "Total pages: 4,565,305\n",
      "Article pages: 2,114,370\n",
      "Redirect pages: 2,450,935\n",
      "Total article lenght: 18,626,439,760\n",
      "Elapsed time: 1:01:29.55\n",
      "4,600,000\n",
      "4,700,000\n",
      "4,800,000\n",
      "4,900,000\n",
      "File processed: enwiki-20180620-pages-meta-current19-p16120543p17620543.xml\n",
      "Total pages: 4,957,962\n",
      "Article pages: 2,226,758\n",
      "Redirect pages: 2,731,204\n",
      "Total article lenght: 19,332,147,085\n",
      "Elapsed time: 1:05:26.37\n",
      "5,000,000\n",
      "5,100,000\n",
      "5,200,000\n",
      "File processed: enwiki-20180620-pages-meta-current19-p17620543p18754735.xml\n",
      "Total pages: 5,205,366\n",
      "Article pages: 2,334,337\n",
      "Redirect pages: 2,871,029\n",
      "Total article lenght: 19,921,622,076\n",
      "Elapsed time: 1:08:35.33\n",
      "5,300,000\n",
      "5,400,000\n",
      "5,500,000\n",
      "File processed: enwiki-20180620-pages-meta-current20-p18754736p20254736.xml\n",
      "Total pages: 5,563,926\n",
      "Article pages: 2,478,259\n",
      "Redirect pages: 3,085,667\n",
      "Total article lenght: 20,717,051,113\n",
      "Elapsed time: 1:12:29.69\n",
      "5,600,000\n",
      "5,700,000\n",
      "File processed: enwiki-20180620-pages-meta-current20-p20254736p21222156.xml\n",
      "Total pages: 5,758,971\n",
      "Article pages: 2,564,310\n",
      "Redirect pages: 3,194,661\n",
      "Total article lenght: 21,220,867,597\n",
      "Elapsed time: 1:14:57.11\n",
      "5,800,000\n",
      "5,900,000\n",
      "6,000,000\n",
      "6,100,000\n",
      "File processed: enwiki-20180620-pages-meta-current21-p21222158p22722158.xml\n",
      "Total pages: 6,117,342\n",
      "Article pages: 2,719,470\n",
      "Redirect pages: 3,397,872\n",
      "Total article lenght: 22,037,870,176\n",
      "Elapsed time: 1:20:28.95\n",
      "6,200,000\n",
      "6,300,000\n",
      "6,400,000\n",
      "File processed: enwiki-20180620-pages-meta-current21-p22722158p23927983.xml\n",
      "Total pages: 6,440,869\n",
      "Article pages: 2,838,204\n",
      "Redirect pages: 3,602,665\n",
      "Total article lenght: 22,727,752,347\n",
      "Elapsed time: 1:24:26.60\n",
      "6,500,000\n",
      "6,600,000\n",
      "6,700,000\n",
      "File processed: enwiki-20180620-pages-meta-current22-p23927984p25427984.xml\n",
      "Total pages: 6,781,139\n",
      "Article pages: 2,982,042\n",
      "Redirect pages: 3,799,097\n",
      "Total article lenght: 23,566,211,008\n",
      "Elapsed time: 1:29:03.83\n",
      "6,800,000\n",
      "6,900,000\n",
      "7,000,000\n",
      "File processed: enwiki-20180620-pages-meta-current22-p25427984p26823660.xml\n",
      "Total pages: 7,085,352\n",
      "Article pages: 3,104,978\n",
      "Redirect pages: 3,980,374\n",
      "Total article lenght: 24,302,988,455\n",
      "Elapsed time: 1:32:53.70\n",
      "7,100,000\n",
      "7,200,000\n",
      "7,300,000\n",
      "7,400,000\n",
      "File processed: enwiki-20180620-pages-meta-current23-p26823661p28323661.xml\n",
      "Total pages: 7,446,305\n",
      "Article pages: 3,244,052\n",
      "Redirect pages: 4,202,253\n",
      "Total article lenght: 25,106,918,973\n",
      "Elapsed time: 1:36:58.63\n",
      "7,500,000\n",
      "7,600,000\n",
      "7,700,000\n",
      "7,800,000\n",
      "File processed: enwiki-20180620-pages-meta-current23-p28323661p29823661.xml\n",
      "Total pages: 7,824,804\n",
      "Article pages: 3,366,857\n",
      "Redirect pages: 4,457,947\n",
      "Total article lenght: 25,773,099,345\n",
      "Elapsed time: 1:40:51.41\n",
      "7,900,000\n",
      "File processed: enwiki-20180620-pages-meta-current23-p29823661p30503449.xml\n",
      "Total pages: 7,991,015\n",
      "Article pages: 3,412,579\n",
      "Redirect pages: 4,578,436\n",
      "Total article lenght: 26,053,361,042\n",
      "Elapsed time: 1:42:36.67\n",
      "8,000,000\n",
      "8,100,000\n",
      "8,200,000\n",
      "8,300,000\n",
      "File processed: enwiki-20180620-pages-meta-current24-p30503451p32003451.xml\n",
      "Total pages: 8,326,592\n",
      "Article pages: 3,544,598\n",
      "Redirect pages: 4,781,994\n",
      "Total article lenght: 26,917,061,639\n",
      "Elapsed time: 1:46:46.43\n",
      "8,400,000\n",
      "8,500,000\n",
      "8,600,000\n",
      "File processed: enwiki-20180620-pages-meta-current24-p32003451p33503451.xml\n",
      "Total pages: 8,655,615\n",
      "Article pages: 3,674,359\n",
      "Redirect pages: 4,981,256\n",
      "Total article lenght: 27,696,875,548\n",
      "Elapsed time: 1:50:56.62\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8,700,000\n",
      "File processed: enwiki-20180620-pages-meta-current24-p33503451p33952815.xml\n",
      "Total pages: 8,751,017\n",
      "Article pages: 3,713,753\n",
      "Redirect pages: 5,037,264\n",
      "Total article lenght: 27,929,477,395\n",
      "Elapsed time: 1:52:03.19\n",
      "8,800,000\n",
      "8,900,000\n",
      "9,000,000\n",
      "File processed: enwiki-20180620-pages-meta-current25-p33952816p35452816.xml\n",
      "Total pages: 9,067,423\n",
      "Article pages: 3,827,235\n",
      "Redirect pages: 5,240,188\n",
      "Total article lenght: 28,612,133,364\n",
      "Elapsed time: 1:55:42.23\n",
      "9,100,000\n",
      "9,200,000\n",
      "9,300,000\n",
      "9,400,000\n",
      "File processed: enwiki-20180620-pages-meta-current25-p35452816p36952816.xml\n",
      "Total pages: 9,432,156\n",
      "Article pages: 3,978,432\n",
      "Redirect pages: 5,453,724\n",
      "Total article lenght: 29,457,358,030\n",
      "Elapsed time: 1:59:38.57\n",
      "9,500,000\n",
      "9,600,000\n",
      "File processed: enwiki-20180620-pages-meta-current25-p36952816p38067202.xml\n",
      "Total pages: 9,687,998\n",
      "Article pages: 4,074,048\n",
      "Redirect pages: 5,613,950\n",
      "Total article lenght: 30,032,740,010\n",
      "Elapsed time: 2:02:29.90\n",
      "9,700,000\n",
      "9,800,000\n",
      "9,900,000\n",
      "10,000,000\n",
      "File processed: enwiki-20180620-pages-meta-current26-p38067203p39567203.xml\n",
      "Total pages: 10,051,357\n",
      "Article pages: 4,205,655\n",
      "Redirect pages: 5,845,702\n",
      "Total article lenght: 30,850,078,384\n",
      "Elapsed time: 2:06:18.66\n",
      "10,100,000\n",
      "10,200,000\n",
      "10,300,000\n",
      "File processed: enwiki-20180620-pages-meta-current26-p39567203p41067203.xml\n",
      "Total pages: 10,389,514\n",
      "Article pages: 4,346,420\n",
      "Redirect pages: 6,043,094\n",
      "Total article lenght: 31,691,960,464\n",
      "Elapsed time: 2:10:27.12\n",
      "10,400,000\n",
      "10,500,000\n",
      "10,600,000\n",
      "10,700,000\n",
      "File processed: enwiki-20180620-pages-meta-current26-p41067203p42567203.xml\n",
      "Total pages: 10,729,106\n",
      "Article pages: 4,487,348\n",
      "Redirect pages: 6,241,758\n",
      "Total article lenght: 32,515,735,202\n",
      "Elapsed time: 2:14:23.73\n",
      "File processed: enwiki-20180620-pages-meta-current26-p42567203p42663461.xml\n",
      "Total pages: 10,750,227\n",
      "Article pages: 4,495,756\n",
      "Redirect pages: 6,254,471\n",
      "Total article lenght: 32,567,509,403\n",
      "Elapsed time: 2:14:41.22\n",
      "10,800,000\n",
      "10,900,000\n",
      "11,000,000\n",
      "11,100,000\n",
      "File processed: enwiki-20180620-pages-meta-current27-p42663462p44163462.xml\n",
      "Total pages: 11,125,958\n",
      "Article pages: 4,635,752\n",
      "Redirect pages: 6,490,206\n",
      "Total article lenght: 33,394,875,528\n",
      "Elapsed time: 2:18:45.93\n",
      "11,200,000\n",
      "11,300,000\n",
      "11,400,000\n",
      "File processed: enwiki-20180620-pages-meta-current27-p44163462p45663462.xml\n",
      "Total pages: 11,411,108\n",
      "Article pages: 4,759,128\n",
      "Redirect pages: 6,651,980\n",
      "Total article lenght: 34,131,725,526\n",
      "Elapsed time: 2:22:38.97\n",
      "11,500,000\n",
      "11,600,000\n",
      "File processed: enwiki-20180620-pages-meta-current27-p45663462p47163462.xml\n",
      "Total pages: 11,647,130\n",
      "Article pages: 4,849,785\n",
      "Redirect pages: 6,797,345\n",
      "Total article lenght: 34,715,266,000\n",
      "Elapsed time: 2:26:14.75\n",
      "11,700,000\n",
      "11,800,000\n",
      "11,900,000\n",
      "File processed: enwiki-20180620-pages-meta-current27-p47163462p48663462.xml\n",
      "Total pages: 11,944,882\n",
      "Article pages: 4,965,326\n",
      "Redirect pages: 6,979,556\n",
      "Total article lenght: 35,427,751,664\n",
      "Elapsed time: 2:30:09.77\n",
      "12,000,000\n",
      "12,100,000\n",
      "12,200,000\n",
      "12,300,000\n",
      "File processed: enwiki-20180620-pages-meta-current27-p48663462p50163462.xml\n",
      "Total pages: 12,314,095\n",
      "Article pages: 5,074,838\n",
      "Redirect pages: 7,239,257\n",
      "Total article lenght: 36,112,970,570\n",
      "Elapsed time: 2:34:05.43\n",
      "12,400,000\n",
      "12,500,000\n",
      "12,600,000\n",
      "File processed: enwiki-20180620-pages-meta-current27-p50163462p51663462.xml\n",
      "Total pages: 12,656,017\n",
      "Article pages: 5,196,852\n",
      "Redirect pages: 7,459,165\n",
      "Total article lenght: 36,859,668,414\n",
      "Elapsed time: 2:38:55.51\n",
      "12,700,000\n",
      "12,800,000\n",
      "12,900,000\n",
      "File processed: enwiki-20180620-pages-meta-current27-p51663462p53163462.xml\n",
      "Total pages: 12,948,733\n",
      "Article pages: 5,308,725\n",
      "Redirect pages: 7,640,008\n",
      "Total article lenght: 37,521,046,837\n",
      "Elapsed time: 2:44:31.73\n",
      "13,000,000\n",
      "13,100,000\n",
      "13,200,000\n",
      "File processed: enwiki-20180620-pages-meta-current27-p53163462p54663462.xml\n",
      "Total pages: 13,279,116\n",
      "Article pages: 5,429,149\n",
      "Redirect pages: 7,849,967\n",
      "Total article lenght: 38,239,784,711\n",
      "Elapsed time: 2:48:32.94\n",
      "13,300,000\n",
      "13,400,000\n",
      "13,500,000\n",
      "13,600,000\n",
      "File processed: enwiki-20180620-pages-meta-current27-p54663462p56163462.xml\n",
      "Total pages: 13,632,367\n",
      "Article pages: 5,531,962\n",
      "Redirect pages: 8,100,405\n",
      "Total article lenght: 38,864,662,227\n",
      "Elapsed time: 2:52:14.47\n",
      "13,700,000\n",
      "13,800,000\n",
      "13,900,000\n",
      "File processed: enwiki-20180620-pages-meta-current27-p56163462p57663462.xml\n",
      "Total pages: 13,967,798\n",
      "Article pages: 5,665,214\n",
      "Redirect pages: 8,302,584\n",
      "Total article lenght: 39,576,523,287\n",
      "Elapsed time: 2:56:10.99\n",
      "File processed: enwiki-20180620-pages-meta-current27-p57663462p57726175.xml\n",
      "Total pages: 13,981,960\n",
      "Article pages: 5,669,870\n",
      "Redirect pages: 8,312,090\n",
      "Total article lenght: 39,601,777,350\n",
      "Elapsed time: 2:56:21.28\n",
      "Total pages: 13,981,960\n",
      "Article pages: 5,669,870\n",
      "Redirect pages: 8,312,090\n",
      "Total article lenght: 39,601,777,350\n",
      "Elapsed time: 2:56:21.28\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "regex_links = re.compile(r\"\\[\\[(?P<article>(?!.*?\\:).*?)(?:\\{\\{.*\\}\\})?(?:\\|(?P<text>(?!.*?\\:).*?))?\\]\\]\")\n",
    "for WikiXML in WIKI_FILENAMES:\n",
    "    pathWikiXML = os.path.join(PATH_WIKI_XML, WikiXML)\n",
    "    pathArticles = os.path.join(PATH_WIKI_OUT, WikiXML+\"_art.csv\")\n",
    "    pathArticlesRedirect = os.path.join(PATH_WIKI_OUT, WikiXML+\"_red.csv\")\n",
    "    with codecs.open(pathArticles, \"w\", ENCODING) as articlesFH, \\\n",
    "    codecs.open(pathArticlesRedirect, \"w\", ENCODING) as redirectFH:\n",
    "        articlesWriter = csv.writer(articlesFH, quoting=csv.QUOTE_MINIMAL)\n",
    "        articlesWriter.writerow(['id', 'title', 'text_len', 'link_pos', 'link_val', 'link_txt']) \n",
    "        redirectWriter = csv.writer(redirectFH, quoting=csv.QUOTE_MINIMAL)\n",
    "        redirectWriter.writerow(['id', 'title', 'redirect'])\n",
    "        for event, elem in etree.iterparse(pathWikiXML, events=('start', 'end')):\n",
    "            tname = strip_tag_name(elem.tag)\n",
    "\n",
    "            if event == 'start':\n",
    "                if tname == 'page':\n",
    "                    title = ''\n",
    "                    id = -1\n",
    "                    redirect = ''\n",
    "                    inrevision = False\n",
    "                    ns = 0\n",
    "                    article_text_len = 0\n",
    "                    links = []\n",
    "                elif tname == 'revision':\n",
    "                    # Do not pick up on revision id's\n",
    "                    inrevision = True\n",
    "            else:\n",
    "                if tname == 'title':\n",
    "                    title = elem.text\n",
    "                elif tname == 'id' and not inrevision:\n",
    "                    id = int(elem.text)\n",
    "                elif tname == 'redirect':\n",
    "                    redirect = elem.attrib['title']\n",
    "                elif tname == 'ns':\n",
    "                    ns = int(elem.text)\n",
    "                elif tname == 'page' and ns == 0:\n",
    "                    totalCount += 1\n",
    "                    total_article_text_len += article_text_len                        \n",
    "                    if len(redirect) == 0:\n",
    "                        articleCount += 1\n",
    "                        if len(links) == 0:\n",
    "                            articlesWriter.writerow([id, title, article_text_len, 0, \"\", \"\"])\n",
    "                        for link in links:\n",
    "                            articlesWriter.writerow([id, title, article_text_len, link[0], link[1], link[2]])\n",
    "                    else:\n",
    "                        redirectCount += 1\n",
    "                        redirectWriter.writerow([id, title, redirect])\n",
    "\n",
    "                    if totalCount > 1 and (totalCount % 100000) == 0:\n",
    "                        print(\"{:,}\".format(totalCount))\n",
    "                elif tname == 'text' and elem.text != None:\n",
    "                    article_text_len = len(elem.text)\n",
    "                    for match in regex_links.finditer(elem.text):    \n",
    "                        link_pos = match.start()\n",
    "                        link_title = match.group(\"article\")\n",
    "                        link_title = link_title.replace(\"&nbsp;\", \" \")\n",
    "                        link_title = link_title.replace(\"&ndash;\", \"-\")\n",
    "                        link_title = link_title.replace(\"&mdash;\", \"—\")\n",
    "                        link_title = link_title.replace(\"%20\", \" \")\n",
    "                        \n",
    "                        link_text = match.group(\"text\")\n",
    "                        links.append((link_pos,link_title,link_text))\n",
    "\n",
    "\n",
    "                elem.clear()\n",
    "    \n",
    "    fn_in = pathArticles\n",
    "    with open(fn_in, 'rb') as f_in, gzip.open(fn_in+'.gz', 'wb') as f_out:\n",
    "        f_out.writelines(f_in)\n",
    "    os.remove(fn_in)\n",
    "    \n",
    "    fn_in = pathArticlesRedirect\n",
    "    with open(fn_in, 'rb') as f_in, gzip.open(fn_in+'.gz', 'wb') as f_out:\n",
    "        f_out.writelines(f_in)\n",
    "    os.remove(fn_in)\n",
    "    \n",
    "    elapsed_time = time.time() - start_time\n",
    "    print(\"File processed: {}\".format(WikiXML))\n",
    "    print(\"Total pages: {:,}\".format(totalCount))\n",
    "    print(\"Article pages: {:,}\".format(articleCount))\n",
    "    print(\"Redirect pages: {:,}\".format(redirectCount))\n",
    "    print(\"Total article lenght: {:,}\".format(total_article_text_len))\n",
    "    print(\"Elapsed time: {}\".format(hms_string(elapsed_time)))\n",
    "\n",
    "elapsed_time = time.time() - start_time\n",
    "\n",
    "print(\"Total pages: {:,}\".format(totalCount))\n",
    "print(\"Article pages: {:,}\".format(articleCount))\n",
    "print(\"Redirect pages: {:,}\".format(redirectCount))\n",
    "print(\"Total article lenght: {:,}\".format(total_article_text_len))\n",
    "print(\"Elapsed time: {}\".format(hms_string(elapsed_time)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
